-
-
Notifications
You must be signed in to change notification settings - Fork 4.2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: Keep a record of stale owners to help better debug poor test isolation #20513
base: main
Are you sure you want to change the base?
Conversation
…s at AB so we can move off the patch ember-source, and that upgrades become easier
Additional context, this is what we do with that map hooks.beforeEach(function (assert) {
const owner = this.owner;
OWNER_MAP.set(guidFor(owner), {
testId: assert.test.testId,
name: assert.test.module.name + ' | ' + assert.test.testName,
});
}); This makes it much easier to trace down what test kicked off some async that leaked into other tests and caused a failure later. |
in ember-qunit, we can pre-wire that in a QUnit.startStart hook. The code in this PR will need to be resilient to a test-adapter forgetting / opting out of "container test-reporting" (what I'm calling this feature) To do that though, we need to get the |
Potentially useful reference code along the same lines, pulled straight out of the big app at LinkedIn (which @rwjblue and @nlfurniss and I built earlier this year)—note that it's super hacky in a couple ways, particularly around the way it does shenanigans to get an actual class name into place for the owner-leak-detector.js/* eslint-disable no-restricted-syntax */
/* global WeakRef, gc */
import QUnit from 'qunit';
import Application from '@ember/application';
import Engine from '@ember/engine';
import * as ApplicationInstanceModule from '@ember/application/instance';
const HAS_GC = typeof gc === 'function';
let OWNER_REFS = [];
async function fullyGC() {
await gc({ type: 'major', execution: 'async' });
await gc({ type: 'major', execution: 'async' });
await gc({ type: 'major', execution: 'async' });
}
async function checkForRetainedOwners() {
if (!HAS_GC) {
throw new Error(
"You're running Owner leak detection but GC is not enabled!"
);
}
await fullyGC();
const testNameToRetainedOwner = new Map();
for (let i = 0; i < OWNER_REFS.length; i++) {
const [ownerRef, testName] = OWNER_REFS[i];
const owner = ownerRef.deref();
if (owner !== undefined) {
let owners = testNameToRetainedOwner.get(testName);
if (owners === undefined) {
owners = [];
testNameToRetainedOwner.set(testName, owners);
}
owners.push(owner);
}
}
OWNER_REFS = [];
return testNameToRetainedOwner;
}
function getOwnerMetadata(owner) {
if (owner.mountPoint) {
return `Engine [mounted at \`/${owner.mountPoint}\`]`;
}
return `Application`;
}
function getTestName() {
const currentTest = QUnit.config.current;
return `${currentTest.module.name}: ${currentTest.testName}`;
}
function trackOwner(owner) {
OWNER_REFS.push([new WeakRef(owner), getTestName()]);
}
function applyMonkeyPatchesToCaptureOwners() {
// eslint-disable-next-line global-require, no-undef
require('voyager-web/instance-initializers/auto-engine-dependencies');
// The block below is done to add a class name to the ApplicationInstanceInstance that isn't present in Ember versions less than 4.x.
// This class name makes it easier to find the owner (needle) in the haystack.
// This can be removed once the Ember 4.x upgrade is done
const ApplicationInstance = ApplicationInstanceModule.default;
ApplicationInstance.proto();
class VWebApplicationInstance extends ApplicationInstanceModule.default {}
// eslint-disable-next-line no-import-assign
ApplicationInstanceModule.default = VWebApplicationInstance;
Application.proto(); // This is needed to setup the prototype in Ember 3.28
const originalBuildApplicationInstance = Application.prototype.buildInstance;
function ApplicationBuildInstanceOverride(options) {
const owner = originalBuildApplicationInstance.call(this, options);
trackOwner(owner);
return owner;
}
ApplicationBuildInstanceOverride.isOwnerCaptureMonkeyPatch = true;
Engine.proto(); // This is needed to setup the prototype in Ember 3.28
const originalBuildEngineInstance = Engine.prototype.buildInstance;
function EngineBuildInstanceOverride(options) {
const owner = originalBuildEngineInstance.call(this, options);
trackOwner(owner);
return owner;
}
EngineBuildInstanceOverride.isOwnerCaptureMonkeyPatch = true;
if (!originalBuildApplicationInstance.isOwnerCaptureMonkeyPatch) {
// This is needed only as long as we have this in Voyager; when it is
// upstreamed into open source libraries we will not need the monkey-patch.
// eslint-disable-next-line @linkedin/pemberly/require-read-only-prototypes
Application.prototype.buildInstance = ApplicationBuildInstanceOverride;
}
if (!originalBuildEngineInstance.isOwnerCaptureMonkeyPatch) {
// This is needed only as long as we have this in Voyager; when it is
// upstreamed into open source libraries we will not need the monkey-patch.
// eslint-disable-next-line @linkedin/pemberly/require-read-only-prototypes
Engine.prototype.buildInstance = EngineBuildInstanceOverride;
}
}
function ensureTestReleasesTestEnvironment(callback = () => {}) {
QUnit.on('testStart', () => {
const currentTest = QUnit.config.current;
const { finish } = currentTest;
currentTest.finish = async function () {
await callback();
return finish.apply(this, arguments);
};
});
}
/**
After each test is completed, check for owner leaks. This will be the best
ergonomics (the actual test that leaks will fail), but due to running GC many
times per test, will be the slowest mechanism.
*/
export function setupPerTestLeakDetection() {
applyMonkeyPatchesToCaptureOwners();
ensureTestReleasesTestEnvironment(async function () {
const currentTest = QUnit.config.current;
for (const [, owners] of await checkForRetainedOwners()) {
for (const owner of owners) {
currentTest.expected++;
currentTest.assert.pushResult({
result: false,
message: `Leaked ${getOwnerMetadata(owner)}`,
});
}
}
});
}
let hasDonePerModuleLeakDetection = false;
/**
After each module is completed, check for owner leaks. This will be faster than
checking for each test, but slower than checking once at the end of all tests.
*/
export function setupPerModuleLeakDetection() {
applyMonkeyPatchesToCaptureOwners();
ensureTestReleasesTestEnvironment();
QUnit.on('suiteEnd', () => {
if (hasDonePerModuleLeakDetection) return;
hasDonePerModuleLeakDetection = true;
QUnit.module('[OWNER LEAK DETECTION]', function () {
const testBody = async function (assert) {
assert.expect(0);
for (const [testName, owner] of await checkForRetainedOwners()) {
assert.pushResult({
result: false,
message: `Leaked ${getOwnerMetadata(owner)} from ${testName}`,
});
}
};
// Opts the user into the `OWNER LEAK DETECTION` module regardless of filter or module selected.
testBody.validTest = true;
QUnit.test('There should be zero leaked Owners', testBody);
});
});
}
let hasCheckedOwnerLeaks = false;
/**
After all tests have been ran, check for any owner leaks. This will be the fastest
mechanism, but also will be ran the least often (you won't get feedback until all
tests have completed).
*/
export function setupAfterAllTestsOwnerLeakDetection() {
applyMonkeyPatchesToCaptureOwners();
ensureTestReleasesTestEnvironment();
// Due to details of how QUnit functions (see https://github.com/qunitjs/qunit/pull/1629)
// we cannot enqueue new tests if we use `runEnd` which is basically what we want (i.e. "when all tests are done run this callback")
//
// Instead we use `suiteEnd` and check `QUnit.config.queue` which indicates the number of tests remaining to be ran
// when `QUnit.config.queue` gets to `0` and `suiteEnd` is running we are finished with all tests **but** `runEnd` / `QUnit.done()` hasn't ran yet so we can still emit new tests
QUnit.on('suiteEnd', () => {
if (QUnit.config.queue.length !== 0) {
return;
}
if (hasCheckedOwnerLeaks) return;
hasCheckedOwnerLeaks = true;
QUnit.module(`[OWNER LEAK DETECTION]`, function () {
const testBody = async function (assert) {
assert.expect(0);
const leakedOwners = await checkForRetainedOwners();
for (const [testName, owners] of leakedOwners) {
for (const owner of owners) {
assert.pushResult({
result: false,
message: `${testName}: Leaked ${getOwnerMetadata(owner)}`,
});
}
}
};
// Opts the user into the `OWNER LEAK DETECTION` module regardless of filter or module selected.
testBody.validTest = true;
QUnit.test('There should be zero leaked Owners', testBody);
});
});
} |
@chriskrycho out of curiosity, how have you enabled gc? All of the v8 flags that are supposedly required have not worked for us to activate the feature any more. We're starting chrome via testem with this flag currently Using a WeakRef for this is clever, though I will say the approach I've taken for this at AB that Preston is looking to get supported here is significantly cheaper and works for leaks that don't retain owner as well. |
Is this still relevant? It is a draft PR |
yeah, it's a problematic test debugging improvement, but needs more thought as to implementation rollout -- it's part of a goal I have to remove internal patches :_| |
This code comes from some internal code that is meant to help track down leaky tests and identify which tests are leaking.
This is useful code, but the patch is complicated enough where upgrading is a bit of a pain, so I'm putting this PR up in collaboration with @runspired and @wagenet to help upstream this ability, and help make it more robust for broader adoption.
Notes:
Map
of owners => test infos.in the internal code, there is:
Map
to aWeakMap
, so that memory is automatically freed when owner references has no more... references.Everything is open to discussion of course. I don't think this needs an RFC, because it could be considered a bugfix of how we handle test isolation (and booting / destroying multiple applications), but it is still worth discussion design of this behavior (I also considered most things that are DX enhancements to rough edges of a library/framework to be bugfixes in general -- the bug being fixed is suboptimal error comprehension).
The patch I'm trying to get rid of